import scrapy
from acad_horizon_spiders.utils import jsonbd_util

""" 

爬取网站基本信息，如主页各会议，年份，查看各会议的所有文章的link等

"""
class BasicMsgSpider(scrapy.Spider):
    name = "basic_msg"
    allowed_domains = ["openaccess.thecvf.com"]
    start_urls = ["https://openaccess.thecvf.com/menu",'https://openaccess.thecvf.com/menu_other.html']
    # 所有会议类型集合
    available_meeting = set()
    # 所有会议年份集合
    available_time = set()
    # cvf网站纪律的所有会议主页链接集合
    cvf_meeting_main_link= []

    

    def parse(self, response):

        self.cvf_meeting_main_link+=response.xpath('//dd/a[1]/@href').getall()
        # ECCV会议相关信息，我们在 ecva.net 中获取，所以此处去除cvf中的ECCV href
        if  '/ECCV2018' in self.cvf_meeting_main_link:
            self.cvf_meeting_main_link.remove('/ECCV2018')
        if  "https://www.ecva.net/papers.php" in self.cvf_meeting_main_link:
            self.cvf_meeting_main_link.remove("https://www.ecva.net/papers.php")

        time_and_meeting_list= response.xpath('//dd')
        self.available_meeting.update({ sele.xpath('./text()').get().split(None,2)[0] for num,sele in enumerate(time_and_meeting_list)})
        self.available_time.update({ sele.xpath('./text()').get().split(None,2)[1][0:-1] for num,sele in enumerate(time_and_meeting_list) })

        temp_time = list(self.available_time)
        temp_time.sort()

        temp_meeting= list(self.available_meeting)
        temp_meeting.sort()
        d = {'available_meeting':temp_meeting,'available_time':temp_time ,'cvf_meeting_main_link':self.cvf_meeting_main_link}
        jsonbd_util.update(d)
        yield 

    